# coding:utf-8
import Queue,threading
import redis
from Proxy_Verify import *


class ThreadStop(threading.Thread):
    _while = True
    def __init__(self, group=None, target=None, name=None,
                 args=(), kwargs=None, verbose=None):
        threading.Thread.__init__(self,group,target,name,args,kwargs,verbose)
    def stop(self):
        self._while = False
#代理验证类
class ThreadUrl(ThreadStop):

    """Threaded Url Grab"""
    def __init__(self, queue,Vaildqueue,NotVaildqueue,cache,reader):
        ThreadStop.__init__(self)
        self.queue = queue
        self.Vaildqueue = Vaildqueue
        self.NotVaildqueue = NotVaildqueue
        self.cache = cache
        self.reader = reader

    def ProxyVaild(self,host,hosttype,proxy_url,mstimeout):
        a = ProxyVaild()
        a.SetProxy(host)
        a.SetProxyType(hosttype)
        a.SetCallBackVaildUrl(proxy_url)
        a.SetTimeOut(mstimeout)
        a.SetGeoIp(self.reader)
        a.SetBody(True)
        return a.Run()

    def run(self):
        proxy_url = self.cache.get('proxy_url')
        mstimeout = int(self.cache.get('proxy_time_ms_out'))
        proxy_vaild_again = int(self.cache.get('proxy_vaild_again'))
        while self._while:
            #grabs host from queue
            host = self.queue.get() #取得一条待验证的代理
            #print host
            for i in range(proxy_vaild_again):
                ReturnValue = self.ProxyVaild(host[0],host[1],proxy_url,mstimeout)
                if len(ReturnValue) > 0 : break
            if host[0] in ReturnValue: #认证  通过返回ip判断是否是活的代理   这句有错误。
                #self.Vaildqueue.put('@'.join(host)) #入库队列
                self.Vaildqueue.put(ReturnValue)
            else:
                self.NotVaildqueue.put('@'.join(host)) #也是入库队列，不过是死的代理库
            #signals to queue job is done
            self.queue.task_done()


#有效代理处理类
class ThreadVaild(ThreadStop):
    def __init__(self, Vaildqueue,Vaildcache):
        ThreadStop.__init__(self)
        self.Vaildqueue = Vaildqueue
        self.Vaildcache = Vaildcache
    def run(self):
        while self._while:
            hostinfo = self.Vaildqueue.get()
            host = '%s@%s'%(hostinfo[0],hostinfo[1])
            hostinfo[0] = str(hostinfo[0]).replace(':','|')
            host_ = '|'.join(str(i) for i in hostinfo)
            print u'成功：%s'%host
            self.Vaildcache.hset('proxy_vaild',host,host_)
            self.Vaildqueue.task_done()


#无效代理处理类
class ThreadNotVaild(ThreadStop):
    def __init__(self, NotVaildqueue,NotVaildcache):
        ThreadStop.__init__(self)
        self.NotVaildqueue = NotVaildqueue
        self.NotVaildcache = NotVaildcache
    def run(self):
        while self._while:
            host = self.NotVaildqueue.get()
            print u'失败：%s'%host
            self.NotVaildcache.sadd('proxy_notvaild',host)
            self.NotVaildcache.hdel('proxy_vaild',host)
            self.NotVaildqueue.task_done()

#有效代理存活数量检测类
class ThreadUrl_(ThreadStop):
    """Threaded Url Grab"""
    def __init__(self ,Vialdqueue_,NotVaildqueue,cache,reader):
        ThreadStop.__init__(self)
        self.Vialdqueue_ = Vialdqueue_
        self.NotVaildqueue = NotVaildqueue
        self.cache = cache
        self.reader = reader
    def ProxyVaild(self,host,port,proxy_url,mstimeout):
        a = ProxyVaild()
        a.SetProxy(host)
        a.SetProxyType(port)
        a.SetCallBackVaildUrl(proxy_url)
        a.SetTimeOut(mstimeout)
        a.SetGeoIp(self.reader)
        a.SetBody(True)
        return a.Run()
    def run(self):
        proxy_url = self.cache.get('proxy_url')
        mstimeout = int(self.cache.get('proxy_time_ms_out'))
        proxy_vaild_again = int(self.cache.get('proxy_vaild_again'))
        while self._while:
            #grabs host from queue
            host = self.Vialdqueue_.get()

            for i in range(proxy_vaild_again):
                ReturnValue = self.ProxyVaild(host[0],host[1],proxy_url,mstimeout)
                if len(ReturnValue) > 2 : break #本来是零的，可是出现错误,可以判断是否联通 如果是小于等于2就是不通
            if host[0] not in ReturnValue:
                self.NotVaildqueue.put('@'.join(host))
                #signals to queue job is done
            else:
                #更新旧数据的相应数据
                oldReturnValue = self.cache.hget('proxy_vaild','@'.join(host)).split('|')
                oldReturnValue[-1] = time.time()
                #print '-----------------------'
                #print oldReturnValue,ReturnValue
                #print '-----------------------'
                oldReturnValue[-3] = ReturnValue[-3]
                host_ = '|'.join(str(i) for i in oldReturnValue)
                self.cache.hset('proxy_vaild','@'.join(host),host_) #进行更新时间处理
            self.Vialdqueue_.task_done()

#任务启动函数
class JobTreadh(ThreadStop):
    def __init__(self,queue,cache):
        ThreadStop.__init__(self)
        self.queue=queue
        self.cache = cache
    #可能需要改 因为这个在线20 30分钟在线抓取一次重复了功能
    def run(self):
        #populate queue with data
        while self._while:
            if self.cache.llen('proxy') > 0: #需要代理库存
                ProxyIpPop = self.cache.lpop('proxy')
                ProxyIp = ProxyIpPop.split('@') #这里有错误
                if self.cache.sismember('proxy_notvaild',ProxyIpPop) is False: #不在垃圾库存里面
                    self.queue.put(ProxyIp) #发送队列入库，认验代理是否有效
            else:
                time.sleep(2) #如果没有库存等待多少时间后继续尝试

#活代理任务启动函数
class JobTreadh_(ThreadStop):
    def __init__(self,Vialdqueue_,cache):
        ThreadStop.__init__(self)
        self.Vialdqueue_=Vialdqueue_
        self.cache = cache
    def run(self):
        #这里需要更改，因为方式变了
        if self.cache.hlen('proxy_vaild') > 0:
            ProxyIpList = self.cache.hkeys('proxy_vaild') #需要验证代理库存
            for i in range(len(ProxyIpList)):
                ProxyIpPop = ProxyIpList.pop()
                ProxyIp = ProxyIpPop.split('@') #这里有错误
                if self.cache.sismember('proxy_notvaild',ProxyIpPop) is False: #不在垃圾库存里面
                    self.Vialdqueue_.put(ProxyIp) #发送队列入库，认验代理是否有效

#少了一个线程



